package com.tzxx.spider.spiderbots.base;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import com.tzxx.spider.core.Constant;
import com.tzxx.spider.domain.Template;

/**
 * @author Created by tzxx
 * @since : 2019/8/5.
 */
public abstract class BaseUrlListSpider implements PageProcessor {

    private final Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    protected final Template template;

    protected BaseUrlListSpider(Template template) {
        this.template = template;
    }

    /**
     * 启动爬虫的逻辑
     *
     * @param threadNumber
     */
    public final void parseLinkList(int threadNumber) {
        Spider.create(this)
                .addUrl(template.getListUrl())
                //开启5个线程抓取
                .thread(threadNumber)
                //启动爬虫
                .run();
    }

    /**
     * 下一页url逻辑
     *
     * @param page
     */
    protected abstract void nextPageUrl(Page page);

    /**
     * 爬取站点的配置信息
     * @return
     */
    @Override
    public final Site getSite() {
        this.site.setUserAgent(Constant.USER_AGENT);
        // cookie具体设计待定
        this.site.addHeader("cookie", template.getRequestBody());
        return this.site;
    }

}
